package googlescrapper;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.IncorrectnessListener;
import com.gargoylesoftware.htmlunit.ScriptException;
import com.gargoylesoftware.htmlunit.SilentCssErrorHandler;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HTMLParserListener;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import javax.swing.JTable;
import javax.swing.JTextPane;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;

/**
 * Google keyword scrapper with proxy
 * @author Shaiful Islam (palash)
 * @email kuvic16@gmail.com
 */
public class GoogleSearching implements Runnable {

    private Thread thread;
    private String threadName;
    private String keyword;
    private javax.swing.JTextPane consle;
    private javax.swing.JTable tblOutput;
    private Object[][] rows = null;
    private List<String> urlList = null;
    private List<Output> outputList = null;
    GoogleScrapper googleScrapper;

    public GoogleSearching(String _threadName, String _keyword, JTextPane _console, Object[][] _rows, List<Output> _output, JTable _tblOutput) {
        threadName = _threadName;
        keyword = _keyword;
        consle = _console;
        rows = _rows;
        urlList = new ArrayList<>();
        outputList = _output;
        if(outputList == null){
            outputList = new ArrayList<Output>();
        }
        tblOutput = _tblOutput;
    }

    @Override
    public void run() {
        this.getGoogleScrapper().logIt("Start google searching for keyword: " + keyword);
        List<String> srcUrl = googleSearching(keyword);
        this.getGoogleScrapper().logIt("SEARCH FINISHED for "+keyword+"! size:" + srcUrl.size() + "\n");
        for(int i=0; i<srcUrl.size(); i++){
            if (this.getGoogleScrapper().getThreadSettings().equalsIgnoreCase("ALL")) {
                AdSearching as1 = new AdSearching(keyword + "adSearch" + i, keyword, consle, rows, outputList, tblOutput, srcUrl.get(i));
                as1.setGoogleScrapper(this.getGoogleScrapper());
                as1.start();
            }else{
                googleAdSearching(srcUrl.get(i), keyword);
            }
        }
    }
    
    private List<String> googleSearching(String keyword){
        List<String> srcUrl = new ArrayList<String>();            
        reSetProxyToSystem();
        setProxyToSystem();                
        try{
            for (int i = 0; i < 3; i++) {
                String url = IConstant.GOOGLE_URL + keyword + "&start=" + i*10;
                
                //this.getGoogleScrapper().logIt("Google searching for keyword " + keyword + "... on " + url + "\n");                
                Document doc = Jsoup.connect(url).userAgent("Mozilla").ignoreHttpErrors(true).timeout(0).get();
                //Document doc = Jsoup.connect(url).userAgent("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2").ignoreHttpErrors(true).timeout(0).get();
                Elements h3s = doc.select("h3[class=r]");
                if(h3s.isEmpty()){
                    this.getGoogleScrapper().logIt("No result found for this keyword");
                    break;
                }
                for (Element h3 : h3s) {
                    Elements links = h3.getElementsByTag("a");
                    for (Element link : links) {
                        String gUrl = link.attr("href");
                        if (gUrl != null && gUrl.length() > 0) {
                            if (gUrl.contains("http")) {
                                gUrl = gUrl.split("&sa=")[0];
                                if (gUrl.split("q=").length > 1) {
                                    gUrl = gUrl.split("q=")[1];
                                }
                                if(!urlList.contains(gUrl)){
                                    if (srcUrl.size() >= 20) {
                                        break;
                                    }
                                    //this.getGoogleScrapper().logIt(gUrl + "\n");
                                    srcUrl.add(gUrl);                                    
                                    urlList.add(gUrl);                                        
                                }
                            }
                        }
                    }
                }
                if(srcUrl.size() >= 20){
                    break;
                }
            }
        }catch(Throwable t){
            this.getGoogleScrapper().logIt("Error during google searching: " + t.getMessage() + "\n");
            System.out.println("Error: " + t);
        }
        return srcUrl;
    }
    
    private void googleAdSearching(String url, String keyword){
        try {
            URI uri = new URI(url);
            Output output = new Output();
            output.setRootDomain(uri.getHost());
            output.setActualDestination(url);
            output.setKeyword(keyword);
            output.setAdFlag("0");

            final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
            configureHtmlUnit(webClient);
            final HtmlPage page = webClient.getPage(url);
            DomNodeList<DomElement> d = page.getElementsByTagName("iframe");
            for (DomElement dd : d) {
                String id = dd.getAttribute("id");
                if (id.contains("google_ads_iframe")) {
                    output.setAdFlag("1");
                    break;
                }
            }
            webClient.closeAllWindows();
            this.getGoogleScrapper().addToOutputList(output);
            //this.getGoogleScrapper().logIt("DONE\n");            
        } catch (Throwable t) {
            this.getGoogleScrapper().logIt("Error during google adSearching: " + t.getMessage() + "\n");
        }
    }
    
    public void configureHtmlUnit(WebClient webClient) {
        webClient.setIncorrectnessListener(new IncorrectnessListener() {

            @Override
            public void notify(String arg0, Object arg1) {
                // TODO Auto-generated method stub

            }
        });
        webClient.setCssErrorHandler(new ErrorHandler() {

            @Override
            public void warning(CSSParseException exception) throws CSSException {
                // TODO Auto-generated method stub

            }

            @Override
            public void fatalError(CSSParseException exception) throws CSSException {
                // TODO Auto-generated method stub

            }

            @Override
            public void error(CSSParseException exception) throws CSSException {
                // TODO Auto-generated method stub

            }
        });
        webClient.setJavaScriptErrorListener(new JavaScriptErrorListener() {

            @Override
            public void timeoutError(HtmlPage arg0, long arg1, long arg2) {
                // TODO Auto-generated method stub

            }

            @Override
            public void scriptException(HtmlPage arg0, ScriptException arg1) {
                // TODO Auto-generated method stub

            }

            @Override
            public void malformedScriptURL(HtmlPage arg0, String arg1, MalformedURLException arg2) {
                // TODO Auto-generated method stub

            }

            @Override
            public void loadScriptError(HtmlPage arg0, URL arg1, Exception arg2) {
                // TODO Auto-generated method stub

            }
        });
        webClient.setHTMLParserListener(new HTMLParserListener() {

            @Override
            public void error(String string, URL url, String string1, int i, int i1, String string2) {

            }

            @Override
            public void warning(String string, URL url, String string1, int i, int i1, String string2) {

            }
        });

        webClient.setCssErrorHandler(new SilentCssErrorHandler());
        webClient.getCookieManager().setCookiesEnabled(false);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setPrintContentOnFailingStatusCode(false);
    }
    
    private void setProxyToSystem(){
        int r = IUtil.randInt(0, rows.length-1);
        System.getProperties().put( "proxySet", "true" );
        if (this.getGoogleScrapper().getProxy().equalsIgnoreCase("HTTP")) {
            System.getProperties().put("http.proxyHost", rows[r][0].toString());
            System.getProperties().put("http.proxyPort", rows[r][1].toString());
        }else{
            System.getProperties().put("https.proxyHost", rows[r][0].toString());
            System.getProperties().put("https.proxyPort", rows[r][1].toString());
        }
    }
    private void reSetProxyToSystem(){
        System.getProperties().put( "proxySet", "false" );
    }
    
    
    public void start() {
        thread = new Thread(this, keyword);
        thread.start();
    }
    
    public GoogleScrapper getGoogleScrapper() {
        return googleScrapper;
    }

    public void setGoogleScrapper(GoogleScrapper googleScrapper) {
        this.googleScrapper = googleScrapper;
    }
}
